RSSL provides implementations of several semi-supervised learning methods. The code can be found on Github. It is described in the following paper: arXiv. More R libraries are used in this notebook for visualization.
Classifiers
There are many classifiers available in the package. The main types are defined below: Least Squares Classifier, Linear Discriminant Analysis, Nearest Mean Classifier, Support Vector Machine and Logistic Regression. They are used together with Self Learning. classifiers is a list of two classifiers, with one supervised classifier and one self learning classifier. Each classifier is a function that accepts 4 arguments: a numeric design matrix of the labeled objects, a factor of labels, a numeric design matrix of unlabeled objects and a factor of labels for the unlabeled objects.
Least Squares Classifier
classifiers_LS <- list(
"LS" = function(X, y, X_u, y_u) {
LeastSquaresClassifier(X, y, lambda = 0)
},
"Self" = function(X, y, X_u, y_u) {
SelfLearning(X, y, X_u, LeastSquaresClassifier)
}
)
Linear Discriminant Analysis
classifiers_LD <- list(
"LD" = function(X, y, X_u, y_u) {
LinearDiscriminantClassifier(X, y)
},
"Self" = function(X, y, X_u, y_u) {
SelfLearning(X, y, X_u, LinearDiscriminantClassifier)
}
)
Nearest Mean Classifier
classifiers_NM <- list(
"NM" = function(X, y, X_u, y_u) {
NearestMeanClassifier(X, y)
},
"Self" = function(X, y, X_u, y_u) {
SelfLearning(X, y, X_u, NearestMeanClassifier)
}
)
Support Vector Machine
classifiers_SVM <- list(
"SVM" = function(X, y, X_u, y_u) {
SVM(X, y)
},
"Self" = function(X, y, X_u, y_u) {
SelfLearning(X, y, X_u, SVM)
}
)
Logistic Regression
classifiers_LR <- list(
"LR" = function(X, y, X_u, y_u) {
LogisticRegression(X, y)
},
"Self" = function(X, y, X_u, y_u) {
SelfLearning(X, y, X_u, LogisticRegression)
}
)
Artificial datasets
RSSL offers many artificial datasets. In the next sections I will generate and visualize them.
2ClassGaussian
data <- generate2ClassGaussian(2000, d = 2, var = 0.6, expected = TRUE)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

LearningCurveSSL evaluates semi-supervised classifiers for different amounts of unlabeled training examples or different fractions of unlabeled vs. labeled examples. This function allows for two different types of learning curves to be generated.
If type="unlabeled", the number of labeled objects remains fixed at the value of n_l, where sizes controls the number of unlabeled objects. n_test controls the number of objects used for the test set, while all remaining objects are used if with_replacement=FALSE in which case objects are drawn without replacement from the input dataset. We make sure each class is represented by at least n_min labeled objects of each class. For n_l, additional options include: “enough” which takes the max of the number of features and 20, max(ncol(X)+5,20), “d” which takes the number of features or “2d” which takes 2 times the number of features.
lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "unlabeled", n_l = "enough", repeats = 3
)
plot(lc)

If type="fraction" the total number of objects remains fixed, while the fraction of labeled objects is changed. frac sets the fractions of labeled objects that should be considered, while test_fraction determines the fraction of the total number of objects left out to serve as the test set.
lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LD, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_NM, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

#lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
# data$Class,
# classifiers = classifiers_SVM, measures = measures,
# type = "fraction", test_fraction = 0.5, repeats = 3
#)
#plot(lc)
lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LR, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

2ClassGaussian (alt)
data <- generate2ClassGaussian(2000, d = 2, var = 0.6, expected = FALSE)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

ABA
data <- generateABA(2000, d = 2, var = 0.6)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

CrescentMoon
data <- generateCrescentMoon(150, 2, 1)
plot(data$X1, data$X2, col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 2:3]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

FourClusters
data <- generateFourClusters(1000, distance = 6, expected = TRUE)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

FourClusters (alt)
data <- generateFourClusters(1000, distance = 6, expected = FALSE)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

ParallelPlanes
data <- generateParallelPlanes(100, 3)
plot(data[, 1], data[, 2], col = data$Class)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

SlicedCookie
data <- generateSlicedCookie(1000, expected = TRUE)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

SlicedCookie (alt)
data <- generateSlicedCookie(1000, expected = FALSE)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

TwoCircles
data <- generateTwoCircles(n = 100, noise_var = 0.2)
plot(data[, 1], data[, 2], col = data$Class, asp = 1)

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

Spirals
data <- generateSpirals(100, sigma = 0.1)
plot3D::scatter3D(data$x, data$y, data$z, col = data$Class)
Warning in `[<-.factor`(`*tmp*`, is.na(Col), value = "white") :
invalid factor level, NA generated

lc <- LearningCurveSSL(as.matrix(data[, 1:2]),
data$Class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", test_fraction = 0.5, repeats = 3
)
plot(lc)

Real datasets
Iris
data("iris")
iris[sample(nrow(iris), 10), ]
pairs(iris[1:4], main = "Iris Scatter Plots", pch = 21, bg = c("red", "green3", "blue")[unclass(iris$Species)])

BpSl <- ggplot(iris, aes(Species, Sepal.Length, fill = Species)) +
geom_boxplot() +
scale_y_continuous("Sepal Length (cm)", breaks = seq(0, 30, by = .5)) +
theme(legend.position = "none")
BpSw <- ggplot(iris, aes(Species, Sepal.Width, fill = Species)) +
geom_boxplot() +
scale_y_continuous("Sepal Width (cm)", breaks = seq(0, 30, by = .5)) +
theme(legend.position = "none")
BpPl <- ggplot(iris, aes(Species, Petal.Length, fill = Species)) +
geom_boxplot() +
scale_y_continuous("Petal Length (cm)", breaks = seq(0, 30, by = .5)) +
theme(legend.position = "none")
BpPw <- ggplot(iris, aes(Species, Petal.Width, fill = Species)) +
geom_boxplot() +
scale_y_continuous("Petal Width (cm)", breaks = seq(0, 30, by = .5)) +
theme(legend.position = "none")
# Plot all visualizations
grid.arrange(BpSl + ggtitle(""),
BpSw + ggtitle(""),
BpPl + ggtitle(""),
BpPw + ggtitle(""),
nrow = 2,
top = textGrob("Iris Box Plots",
gp = gpar(fontsize = 15)
)
)

lc_iris <- LearningCurveSSL(as.matrix(iris[1:4]), iris$Species,
classifiers = classifiers_LS, measures = measures,
type = "fraction", fracs = seq(0.1, 0.8, 0.1),
test_fraction = 0.5, repeats = 3
)
plot(lc_iris)

Spambase

spambase <- read.csv("../data/spambase.csv", header = TRUE, sep = ",")
spambase$class <- as.factor(spambase$class)
spambase[sample(nrow(spambase), 10), ]
lc_spambase <- LearningCurveSSL(as.matrix(spambase[1:57]), spambase$class,
classifiers = classifiers_LS, measures = measures,
type = "fraction", fracs = seq(0.1, 0.8, 0.1),
test_fraction = 0.5, repeats = 3
)
plot(lc_spambase)

LS0tDQp0aXRsZTogJ1JTU0w6IFNlbWktU3VwZXJ2aXNlZCBMZWFybmluZyBpbiBSJw0Kb3V0cHV0Og0KICBodG1sX25vdGVib29rOiANCiAgICB0b2M6IHllcw0KICAgIHRvY19mbG9hdDogeWVzDQogICAgbnVtYmVyX3NlY3Rpb25zOiB5ZXMNCi0tLQ0KDQohW1NlbWktU3VwZXJ2aXNlZCBMZWFybmluZ10oLi4vaW1hZ2VzL3NlbWlfc3VwZXJ2aXNlZF9sZWFybmluZy5wbmcpDQoNClJTU0wgcHJvdmlkZXMgaW1wbGVtZW50YXRpb25zIG9mIHNldmVyYWwgc2VtaS1zdXBlcnZpc2VkIGxlYXJuaW5nIG1ldGhvZHMuIFRoZSBjb2RlIGNhbiBiZSBmb3VuZCBvbiBbR2l0aHViXShodHRwczovL2dpdGh1Yi5jb20vamtyaWp0aGUvUlNTTCkuIEl0IGlzIGRlc2NyaWJlZCBpbiB0aGUgZm9sbG93aW5nIHBhcGVyOiBbYXJYaXZdKGh0dHBzOi8vYXJ4aXYub3JnL3BkZi8xNjEyLjA3OTkzLnBkZikuIE1vcmUgUiBsaWJyYXJpZXMgYXJlIHVzZWQgaW4gdGhpcyBub3RlYm9vayBmb3IgdmlzdWFsaXphdGlvbi4NCg0KYGBge3J9DQpsaWJyYXJ5KFJTU0wpDQpsaWJyYXJ5KGdncGxvdDIpDQpsaWJyYXJ5KGdyaWQpDQpsaWJyYXJ5KGdyaWRFeHRyYSkNCmxpYnJhcnkocGxvdDNEKQ0Kc2V0LnNlZWQoMSkNCmBgYA0KDQojIENsYXNzaWZpZXJzDQoNClRoZXJlIGFyZSBtYW55IGNsYXNzaWZpZXJzIGF2YWlsYWJsZSBpbiB0aGUgcGFja2FnZS4gVGhlIG1haW4gdHlwZXMgYXJlIGRlZmluZWQgYmVsb3c6IExlYXN0IFNxdWFyZXMgQ2xhc3NpZmllciwgTGluZWFyIERpc2NyaW1pbmFudCBBbmFseXNpcywgTmVhcmVzdCBNZWFuIENsYXNzaWZpZXIsIFN1cHBvcnQgVmVjdG9yIE1hY2hpbmUgYW5kIExvZ2lzdGljIFJlZ3Jlc3Npb24uIFRoZXkgYXJlIHVzZWQgdG9nZXRoZXIgd2l0aCBTZWxmIExlYXJuaW5nLiBgY2xhc3NpZmllcnNgIGlzIGEgbGlzdCBvZiB0d28gY2xhc3NpZmllcnMsIHdpdGggb25lIHN1cGVydmlzZWQgY2xhc3NpZmllciBhbmQgb25lIHNlbGYgbGVhcm5pbmcgY2xhc3NpZmllci4gRWFjaA0KY2xhc3NpZmllciBpcyBhIGZ1bmN0aW9uIHRoYXQgYWNjZXB0cyA0IGFyZ3VtZW50czogYQ0KbnVtZXJpYyBkZXNpZ24gbWF0cml4IG9mIHRoZSBsYWJlbGVkIG9iamVjdHMsIGEgZmFjdG9yIG9mIGxhYmVscywNCmEgbnVtZXJpYyBkZXNpZ24gbWF0cml4IG9mIHVubGFiZWxlZCBvYmplY3RzIGFuZCBhIGZhY3RvciBvZg0KbGFiZWxzIGZvciB0aGUgdW5sYWJlbGVkIG9iamVjdHMuDQoNCiMjIExlYXN0IFNxdWFyZXMgQ2xhc3NpZmllcg0KDQohW0xlYXN0IFNxdWFyZXMgQ2xhc3NpZmllcl0oLi4vaW1hZ2VzL0xTLnBuZykNCg0KYGBge3J9DQpjbGFzc2lmaWVyc19MUyA8LSBsaXN0KA0KICAgICJMUyIgPSBmdW5jdGlvbihYLCB5LCBYX3UsIHlfdSkgew0KICAgICAgICBMZWFzdFNxdWFyZXNDbGFzc2lmaWVyKFgsIHksIGxhbWJkYSA9IDApDQogICAgfSwNCiAgICAiU2VsZiIgPSBmdW5jdGlvbihYLCB5LCBYX3UsIHlfdSkgew0KICAgICAgICBTZWxmTGVhcm5pbmcoWCwgeSwgWF91LCBMZWFzdFNxdWFyZXNDbGFzc2lmaWVyKQ0KICAgIH0NCikNCmBgYA0KDQojIyBMaW5lYXIgRGlzY3JpbWluYW50IEFuYWx5c2lzDQoNCiFbTGluZWFyIERpc2NyaW1pbmFudCBBbmFseXNpc10oLi4vaW1hZ2VzL0xELmpwZykNCg0KYGBge3J9DQpjbGFzc2lmaWVyc19MRCA8LSBsaXN0KA0KICAgICJMRCIgPSBmdW5jdGlvbihYLCB5LCBYX3UsIHlfdSkgew0KICAgICAgICBMaW5lYXJEaXNjcmltaW5hbnRDbGFzc2lmaWVyKFgsIHkpDQogICAgfSwNCiAgICAiU2VsZiIgPSBmdW5jdGlvbihYLCB5LCBYX3UsIHlfdSkgew0KICAgICAgICBTZWxmTGVhcm5pbmcoWCwgeSwgWF91LCBMaW5lYXJEaXNjcmltaW5hbnRDbGFzc2lmaWVyKQ0KICAgIH0NCikNCmBgYA0KDQoNCiMjIE5lYXJlc3QgTWVhbiBDbGFzc2lmaWVyDQoNCiFbTmVhcmVzdCBNZWFuIENsYXNzaWZpZXJdKC4uL2ltYWdlcy9OTS5wbmcpDQoNCmBgYHtyfQ0KY2xhc3NpZmllcnNfTk0gPC0gbGlzdCgNCiAgICAiTk0iID0gZnVuY3Rpb24oWCwgeSwgWF91LCB5X3UpIHsNCiAgICAgICAgTmVhcmVzdE1lYW5DbGFzc2lmaWVyKFgsIHkpDQogICAgfSwNCiAgICAiU2VsZiIgPSBmdW5jdGlvbihYLCB5LCBYX3UsIHlfdSkgew0KICAgICAgICBTZWxmTGVhcm5pbmcoWCwgeSwgWF91LCBOZWFyZXN0TWVhbkNsYXNzaWZpZXIpDQogICAgfQ0KKQ0KYGBgDQoNCg0KIyMgU3VwcG9ydCBWZWN0b3IgTWFjaGluZQ0KDQohW1N1cHBvcnQgVmVjdG9yIE1hY2hpbmVdKC4uL2ltYWdlcy9TVk0ucG5nKQ0KDQpgYGB7cn0NCmNsYXNzaWZpZXJzX1NWTSA8LSBsaXN0KA0KICAgICJTVk0iID0gZnVuY3Rpb24oWCwgeSwgWF91LCB5X3UpIHsNCiAgICAgICAgU1ZNKFgsIHkpDQogICAgfSwNCiAgICAiU2VsZiIgPSBmdW5jdGlvbihYLCB5LCBYX3UsIHlfdSkgew0KICAgICAgICBTZWxmTGVhcm5pbmcoWCwgeSwgWF91LCBTVk0pDQogICAgfQ0KKQ0KYGBgDQoNCg0KIyMgTG9naXN0aWMgUmVncmVzc2lvbg0KDQohW0xvZ2lzdGljIFJlZ3Jlc3Npb25dKC4uL2ltYWdlcy9MUi5qcGcpDQoNCmBgYHtyfQ0KY2xhc3NpZmllcnNfTFIgPC0gbGlzdCgNCiAgICAiTFIiID0gZnVuY3Rpb24oWCwgeSwgWF91LCB5X3UpIHsNCiAgICAgICAgTG9naXN0aWNSZWdyZXNzaW9uKFgsIHkpDQogICAgfSwNCiAgICAiU2VsZiIgPSBmdW5jdGlvbihYLCB5LCBYX3UsIHlfdSkgew0KICAgICAgICBTZWxmTGVhcm5pbmcoWCwgeSwgWF91LCBMb2dpc3RpY1JlZ3Jlc3Npb24pDQogICAgfQ0KKQ0KYGBgDQoNCiMgTWVhc3VyZXMNCg0KVGhlcmUgYXJlIGZpdmUgcGVyZm9ybWFuY2UgbWVhc3VyZXMgYXZhaWxhYmxlOiBhY2N1cmFjeSwgZXJyb3IsIHRlc3QgbG9zcywgbGFiZWxlZCBsb3NzIGFuZCB0cmFpbiBsb3NzLiBgbWVhc3VyZXNgIGlzIGEgbGlzdCBvZiBwZXJmb3JtYW5jZSBtZWFzdXJlcyB0aGF0IHdlIHdhbnQgdG8gc2VsZWN0LiBPdXIgYWltIGlzIHRvIGltcHJvdmUgdGhlIGFjY3VyYWN5IG9uIHRoZSB0ZXN0IHNldCAob3IgcmVkdWNlIHRoZSBlcnJvcikuIFdlIGNhbiBsb29rIGF0IHRoZSBsb3NzZXMgb2YgZWFjaCBzcGxpdCB0byBrbm93IGhvdyB0aGUgbW9kZWwgaXMgbGVhcm5pbmcuIFRoZSB0aW1lIGlzIGFsc28gYW4gaW1wb3J0YW50IG1lYXN1cmUgd2hlbiB3b3JraW5nIHdpdGggYmlnIGRhdGFzZXRzLg0KDQohW01lYXN1cmVzXSguLi9pbWFnZXMvbWVhc3VyZXMucG5nKQ0KDQpgYGB7cn0NCm1lYXN1cmVzIDwtIGxpc3QoDQogICAgIkFjY3VyYWN5IiA9IG1lYXN1cmVfYWNjdXJhY3ksDQogICAgIkVycm9yIiA9IG1lYXN1cmVfZXJyb3IsDQogICAgIkxvc3MgVGVzdCIgPSBtZWFzdXJlX2xvc3N0ZXN0LA0KICAgICJMb3NzIExhYmVsZWQiID0gbWVhc3VyZV9sb3NzbGFiLA0KICAgICJMb3NzIFRyYWluIiA9IG1lYXN1cmVfbG9zc3RyYWluDQopDQpgYGANCg0KIyBBcnRpZmljaWFsIGRhdGFzZXRzDQoNClJTU0wgb2ZmZXJzIG1hbnkgYXJ0aWZpY2lhbCBkYXRhc2V0cy4gSW4gdGhlIG5leHQgc2VjdGlvbnMgSSB3aWxsIGdlbmVyYXRlIGFuZCB2aXN1YWxpemUgdGhlbS4NCg0KIVtTaW11bGF0ZWQgRGF0YXNldHMuIEVhY2ggY2FuIGJlIGdlbmVyYXRlZCB1c2luZyBhIGZ1bmN0aW9uIG9mIHRoZSBmb3JtDQpnZW5lcmF0ZURhdGFzZXQsIHdoZXJlIERhdGFzZXQgc2hvdWxkIGJlIHJlcGxhY2VkIGJ5IHRoZSBuYW1lIG9mIHRoZSBkYXRhc2V0LiAoYWx0KSBpbmRpY2F0ZXMNCm5vbi1kZWZhdWx0IHBhcmFtZXRlcnMgd2VyZSB1c2VkIHdoZW4gY2FsbGluZyB0aGUgZnVuY3Rpb24uXSguLi9pbWFnZXMvYXJ0aWZpY2lhbF9kYXRhc2V0cy5wbmcpDQoNCiMjIDJDbGFzc0dhdXNzaWFuDQoNCmBgYHtyfQ0KZGF0YSA8LSBnZW5lcmF0ZTJDbGFzc0dhdXNzaWFuKDIwMDAsIGQgPSAyLCB2YXIgPSAwLjYsIGV4cGVjdGVkID0gVFJVRSkNCnBsb3QoZGF0YVssIDFdLCBkYXRhWywgMl0sIGNvbCA9IGRhdGEkQ2xhc3MsIGFzcCA9IDEpDQpgYGANCg0KYExlYXJuaW5nQ3VydmVTU0xgIGV2YWx1YXRlcyBzZW1pLXN1cGVydmlzZWQgY2xhc3NpZmllcnMgZm9yIGRpZmZlcmVudCBhbW91bnRzIG9mDQp1bmxhYmVsZWQgdHJhaW5pbmcgZXhhbXBsZXMgb3IgZGlmZmVyZW50IGZyYWN0aW9ucyBvZiB1bmxhYmVsZWQNCnZzLiBsYWJlbGVkIGV4YW1wbGVzLiBUaGlzIGZ1bmN0aW9uIGFsbG93cyBmb3IgdHdvIGRpZmZlcmVudCB0eXBlcyBvZiBsZWFybmluZyBjdXJ2ZXMgdG8NCmJlIGdlbmVyYXRlZC4gDQoNCklmIGB0eXBlPSJ1bmxhYmVsZWQiYCwgdGhlIG51bWJlciBvZiBsYWJlbGVkIG9iamVjdHMNCnJlbWFpbnMgZml4ZWQgYXQgdGhlIHZhbHVlIG9mIGBuX2xgLCB3aGVyZSBgc2l6ZXNgIGNvbnRyb2xzIHRoZQ0KbnVtYmVyIG9mIHVubGFiZWxlZCBvYmplY3RzLiBgbl90ZXN0YCBjb250cm9scyB0aGUgbnVtYmVyIG9mDQpvYmplY3RzIHVzZWQgZm9yIHRoZSB0ZXN0IHNldCwgd2hpbGUgYWxsIHJlbWFpbmluZyBvYmplY3RzIGFyZQ0KdXNlZCBpZiBgd2l0aF9yZXBsYWNlbWVudD1GQUxTRWAgaW4gd2hpY2ggY2FzZSBvYmplY3RzIGFyZSBkcmF3bg0Kd2l0aG91dCByZXBsYWNlbWVudCBmcm9tIHRoZSBpbnB1dCBkYXRhc2V0LiBXZSBtYWtlIHN1cmUgZWFjaA0KY2xhc3MgaXMgcmVwcmVzZW50ZWQgYnkgYXQgbGVhc3QgYG5fbWluYCBsYWJlbGVkIG9iamVjdHMgb2YgZWFjaA0KY2xhc3MuIEZvciBgbl9sYCwgYWRkaXRpb25hbCBvcHRpb25zIGluY2x1ZGU6ICJlbm91Z2giIHdoaWNoIHRha2VzDQp0aGUgbWF4IG9mIHRoZSBudW1iZXIgb2YgZmVhdHVyZXMgYW5kIDIwLCBtYXgobmNvbChYKSs1LDIwKSwgImQiDQp3aGljaCB0YWtlcyB0aGUgbnVtYmVyIG9mIGZlYXR1cmVzIG9yICIyZCIgd2hpY2ggdGFrZXMgMiB0aW1lcyB0aGUNCm51bWJlciBvZiBmZWF0dXJlcy4NCg0KYGBge3J9DQpsYyA8LSBMZWFybmluZ0N1cnZlU1NMKGFzLm1hdHJpeChkYXRhWywgMToyXSksDQogICAgZGF0YSRDbGFzcywNCiAgICBjbGFzc2lmaWVycyA9IGNsYXNzaWZpZXJzX0xTLCBtZWFzdXJlcyA9IG1lYXN1cmVzLA0KICAgIHR5cGUgPSAidW5sYWJlbGVkIiwgbl9sID0gImVub3VnaCIsIHJlcGVhdHMgPSAzDQopDQoNCnBsb3QobGMpDQpgYGANCg0KSWYgYHR5cGU9ImZyYWN0aW9uImAgdGhlIHRvdGFsIG51bWJlciBvZiBvYmplY3RzIHJlbWFpbnMgZml4ZWQsDQp3aGlsZSB0aGUgZnJhY3Rpb24gb2YgbGFiZWxlZCBvYmplY3RzIGlzIGNoYW5nZWQuIGBmcmFjYCBzZXRzIHRoZQ0KZnJhY3Rpb25zIG9mIGxhYmVsZWQgb2JqZWN0cyB0aGF0IHNob3VsZCBiZSBjb25zaWRlcmVkLCB3aGlsZQ0KYHRlc3RfZnJhY3Rpb25gIGRldGVybWluZXMgdGhlIGZyYWN0aW9uIG9mIHRoZSB0b3RhbCBudW1iZXIgb2YNCm9iamVjdHMgbGVmdCBvdXQgdG8gc2VydmUgYXMgdGhlIHRlc3Qgc2V0Lg0KDQpgYGB7cn0NCmxjIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KGRhdGFbLCAxOjJdKSwNCiAgICBkYXRhJENsYXNzLA0KICAgIGNsYXNzaWZpZXJzID0gY2xhc3NpZmllcnNfTFMsIG1lYXN1cmVzID0gbWVhc3VyZXMsDQogICAgdHlwZSA9ICJmcmFjdGlvbiIsIHRlc3RfZnJhY3Rpb24gPSAwLjUsIHJlcGVhdHMgPSAzDQopDQoNCnBsb3QobGMpDQpgYGANCg0KYGBge3J9DQpsYyA8LSBMZWFybmluZ0N1cnZlU1NMKGFzLm1hdHJpeChkYXRhWywgMToyXSksDQogICAgZGF0YSRDbGFzcywNCiAgICBjbGFzc2lmaWVycyA9IGNsYXNzaWZpZXJzX0xELCBtZWFzdXJlcyA9IG1lYXN1cmVzLA0KICAgIHR5cGUgPSAiZnJhY3Rpb24iLCB0ZXN0X2ZyYWN0aW9uID0gMC41LCByZXBlYXRzID0gMw0KKQ0KDQpwbG90KGxjKQ0KYGBgDQoNCmBgYHtyfQ0KbGMgPC0gTGVhcm5pbmdDdXJ2ZVNTTChhcy5tYXRyaXgoZGF0YVssIDE6Ml0pLA0KICAgIGRhdGEkQ2xhc3MsDQogICAgY2xhc3NpZmllcnMgPSBjbGFzc2lmaWVyc19OTSwgbWVhc3VyZXMgPSBtZWFzdXJlcywNCiAgICB0eXBlID0gImZyYWN0aW9uIiwgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCikNCg0KcGxvdChsYykNCmBgYA0KDQpgYGB7cn0NCiNsYyA8LSBMZWFybmluZ0N1cnZlU1NMKGFzLm1hdHJpeChkYXRhWywgMToyXSksDQojICAgIGRhdGEkQ2xhc3MsDQojICAgIGNsYXNzaWZpZXJzID0gY2xhc3NpZmllcnNfU1ZNLCBtZWFzdXJlcyA9IG1lYXN1cmVzLA0KIyAgICB0eXBlID0gImZyYWN0aW9uIiwgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCiMpDQoNCiNwbG90KGxjKQ0KYGBgDQoNCmBgYHtyfQ0KbGMgPC0gTGVhcm5pbmdDdXJ2ZVNTTChhcy5tYXRyaXgoZGF0YVssIDE6Ml0pLA0KICAgIGRhdGEkQ2xhc3MsDQogICAgY2xhc3NpZmllcnMgPSBjbGFzc2lmaWVyc19MUiwgbWVhc3VyZXMgPSBtZWFzdXJlcywNCiAgICB0eXBlID0gImZyYWN0aW9uIiwgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCikNCg0KcGxvdChsYykNCmBgYA0KDQojIyAyQ2xhc3NHYXVzc2lhbiAoYWx0KQ0KDQpgYGB7cn0NCmRhdGEgPC0gZ2VuZXJhdGUyQ2xhc3NHYXVzc2lhbigyMDAwLCBkID0gMiwgdmFyID0gMC42LCBleHBlY3RlZCA9IEZBTFNFKQ0KcGxvdChkYXRhWywgMV0sIGRhdGFbLCAyXSwgY29sID0gZGF0YSRDbGFzcywgYXNwID0gMSkNCmBgYA0KDQpgYGB7cn0NCmxjIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KGRhdGFbLCAxOjJdKSwNCiAgICBkYXRhJENsYXNzLA0KICAgIGNsYXNzaWZpZXJzID0gY2xhc3NpZmllcnNfTFMsIG1lYXN1cmVzID0gbWVhc3VyZXMsDQogICAgdHlwZSA9ICJmcmFjdGlvbiIsIHRlc3RfZnJhY3Rpb24gPSAwLjUsIHJlcGVhdHMgPSAzDQopDQoNCnBsb3QobGMpDQpgYGANCg0KDQojIyBBQkENCg0KYGBge3J9DQpkYXRhIDwtIGdlbmVyYXRlQUJBKDIwMDAsIGQgPSAyLCB2YXIgPSAwLjYpDQpwbG90KGRhdGFbLCAxXSwgZGF0YVssIDJdLCBjb2wgPSBkYXRhJENsYXNzLCBhc3AgPSAxKQ0KYGBgDQoNCmBgYHtyfQ0KbGMgPC0gTGVhcm5pbmdDdXJ2ZVNTTChhcy5tYXRyaXgoZGF0YVssIDE6Ml0pLA0KICAgIGRhdGEkQ2xhc3MsDQogICAgY2xhc3NpZmllcnMgPSBjbGFzc2lmaWVyc19MUywgbWVhc3VyZXMgPSBtZWFzdXJlcywNCiAgICB0eXBlID0gImZyYWN0aW9uIiwgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCikNCg0KcGxvdChsYykNCmBgYA0KDQojIyBDcmVzY2VudE1vb24NCg0KYGBge3J9DQpkYXRhIDwtIGdlbmVyYXRlQ3Jlc2NlbnRNb29uKDE1MCwgMiwgMSkNCnBsb3QoZGF0YSRYMSwgZGF0YSRYMiwgY29sID0gZGF0YSRDbGFzcywgYXNwID0gMSkNCmBgYA0KDQpgYGB7cn0NCmxjIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KGRhdGFbLCAyOjNdKSwNCiAgICBkYXRhJENsYXNzLA0KICAgIGNsYXNzaWZpZXJzID0gY2xhc3NpZmllcnNfTFMsIG1lYXN1cmVzID0gbWVhc3VyZXMsDQogICAgdHlwZSA9ICJmcmFjdGlvbiIsIHRlc3RfZnJhY3Rpb24gPSAwLjUsIHJlcGVhdHMgPSAzDQopDQoNCnBsb3QobGMpDQpgYGANCg0KIyMgRm91ckNsdXN0ZXJzDQoNCmBgYHtyfQ0KZGF0YSA8LSBnZW5lcmF0ZUZvdXJDbHVzdGVycygxMDAwLCBkaXN0YW5jZSA9IDYsIGV4cGVjdGVkID0gVFJVRSkNCnBsb3QoZGF0YVssIDFdLCBkYXRhWywgMl0sIGNvbCA9IGRhdGEkQ2xhc3MsIGFzcCA9IDEpDQpgYGANCg0KYGBge3J9DQpsYyA8LSBMZWFybmluZ0N1cnZlU1NMKGFzLm1hdHJpeChkYXRhWywgMToyXSksDQogICAgZGF0YSRDbGFzcywNCiAgICBjbGFzc2lmaWVycyA9IGNsYXNzaWZpZXJzX0xTLCBtZWFzdXJlcyA9IG1lYXN1cmVzLA0KICAgIHR5cGUgPSAiZnJhY3Rpb24iLCB0ZXN0X2ZyYWN0aW9uID0gMC41LCByZXBlYXRzID0gMw0KKQ0KDQpwbG90KGxjKQ0KYGBgDQoNCiMjIEZvdXJDbHVzdGVycyAoYWx0KQ0KDQpgYGB7cn0NCmRhdGEgPC0gZ2VuZXJhdGVGb3VyQ2x1c3RlcnMoMTAwMCwgZGlzdGFuY2UgPSA2LCBleHBlY3RlZCA9IEZBTFNFKQ0KcGxvdChkYXRhWywgMV0sIGRhdGFbLCAyXSwgY29sID0gZGF0YSRDbGFzcywgYXNwID0gMSkNCmBgYA0KDQpgYGB7cn0NCmxjIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KGRhdGFbLCAxOjJdKSwNCiAgICBkYXRhJENsYXNzLA0KICAgIGNsYXNzaWZpZXJzID0gY2xhc3NpZmllcnNfTFMsIG1lYXN1cmVzID0gbWVhc3VyZXMsDQogICAgdHlwZSA9ICJmcmFjdGlvbiIsIHRlc3RfZnJhY3Rpb24gPSAwLjUsIHJlcGVhdHMgPSAzDQopDQoNCnBsb3QobGMpDQpgYGANCg0KIyMgUGFyYWxsZWxQbGFuZXMNCg0KYGBge3J9DQpkYXRhIDwtIGdlbmVyYXRlUGFyYWxsZWxQbGFuZXMoMTAwLCAzKQ0KcGxvdChkYXRhWywgMV0sIGRhdGFbLCAyXSwgY29sID0gZGF0YSRDbGFzcykNCmBgYA0KDQpgYGB7cn0NCmxjIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KGRhdGFbLCAxOjJdKSwNCiAgICBkYXRhJENsYXNzLA0KICAgIGNsYXNzaWZpZXJzID0gY2xhc3NpZmllcnNfTFMsIG1lYXN1cmVzID0gbWVhc3VyZXMsDQogICAgdHlwZSA9ICJmcmFjdGlvbiIsIHRlc3RfZnJhY3Rpb24gPSAwLjUsIHJlcGVhdHMgPSAzDQopDQoNCnBsb3QobGMpDQpgYGANCg0KIyMgU2xpY2VkQ29va2llDQoNCmBgYHtyfQ0KZGF0YSA8LSBnZW5lcmF0ZVNsaWNlZENvb2tpZSgxMDAwLCBleHBlY3RlZCA9IFRSVUUpDQpwbG90KGRhdGFbLCAxXSwgZGF0YVssIDJdLCBjb2wgPSBkYXRhJENsYXNzLCBhc3AgPSAxKQ0KYGBgDQoNCmBgYHtyfQ0KbGMgPC0gTGVhcm5pbmdDdXJ2ZVNTTChhcy5tYXRyaXgoZGF0YVssIDE6Ml0pLA0KICAgIGRhdGEkQ2xhc3MsDQogICAgY2xhc3NpZmllcnMgPSBjbGFzc2lmaWVyc19MUywgbWVhc3VyZXMgPSBtZWFzdXJlcywNCiAgICB0eXBlID0gImZyYWN0aW9uIiwgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCikNCg0KcGxvdChsYykNCmBgYA0KDQojIyBTbGljZWRDb29raWUgKGFsdCkNCg0KYGBge3J9DQpkYXRhIDwtIGdlbmVyYXRlU2xpY2VkQ29va2llKDEwMDAsIGV4cGVjdGVkID0gRkFMU0UpDQpwbG90KGRhdGFbLCAxXSwgZGF0YVssIDJdLCBjb2wgPSBkYXRhJENsYXNzLCBhc3AgPSAxKQ0KYGBgDQoNCmBgYHtyfQ0KbGMgPC0gTGVhcm5pbmdDdXJ2ZVNTTChhcy5tYXRyaXgoZGF0YVssIDE6Ml0pLA0KICAgIGRhdGEkQ2xhc3MsDQogICAgY2xhc3NpZmllcnMgPSBjbGFzc2lmaWVyc19MUywgbWVhc3VyZXMgPSBtZWFzdXJlcywNCiAgICB0eXBlID0gImZyYWN0aW9uIiwgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCikNCg0KcGxvdChsYykNCmBgYA0KDQojIyBUd29DaXJjbGVzDQoNCmBgYHtyfQ0KZGF0YSA8LSBnZW5lcmF0ZVR3b0NpcmNsZXMobiA9IDEwMCwgbm9pc2VfdmFyID0gMC4yKQ0KcGxvdChkYXRhWywgMV0sIGRhdGFbLCAyXSwgY29sID0gZGF0YSRDbGFzcywgYXNwID0gMSkNCmBgYA0KDQpgYGB7cn0NCmxjIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KGRhdGFbLCAxOjJdKSwNCiAgICBkYXRhJENsYXNzLA0KICAgIGNsYXNzaWZpZXJzID0gY2xhc3NpZmllcnNfTFMsIG1lYXN1cmVzID0gbWVhc3VyZXMsDQogICAgdHlwZSA9ICJmcmFjdGlvbiIsIHRlc3RfZnJhY3Rpb24gPSAwLjUsIHJlcGVhdHMgPSAzDQopDQoNCnBsb3QobGMpDQpgYGANCg0KIyMgU3BpcmFscw0KDQpgYGB7cn0NCmRhdGEgPC0gZ2VuZXJhdGVTcGlyYWxzKDEwMCwgc2lnbWEgPSAwLjEpDQpwbG90M0Q6OnNjYXR0ZXIzRChkYXRhJHgsIGRhdGEkeSwgZGF0YSR6LCBjb2wgPSBkYXRhJENsYXNzKQ0KYGBgDQoNCmBgYHtyfQ0KbGMgPC0gTGVhcm5pbmdDdXJ2ZVNTTChhcy5tYXRyaXgoZGF0YVssIDE6Ml0pLA0KICAgIGRhdGEkQ2xhc3MsDQogICAgY2xhc3NpZmllcnMgPSBjbGFzc2lmaWVyc19MUywgbWVhc3VyZXMgPSBtZWFzdXJlcywNCiAgICB0eXBlID0gImZyYWN0aW9uIiwgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCikNCg0KcGxvdChsYykNCmBgYA0KDQojIFJlYWwgZGF0YXNldHMNCg0KIyMgSXJpcw0KDQohW0lyaXNdKC4uL2ltYWdlcy9pcmlzLnBuZykNCg0KYGBge3J9DQpkYXRhKCJpcmlzIikNCmlyaXNbc2FtcGxlKG5yb3coaXJpcyksIDEwKSwgXQ0KYGBgDQoNCmBgYHtyfQ0KcGFpcnMoaXJpc1sxOjRdLCBtYWluID0gIklyaXMgU2NhdHRlciBQbG90cyIsIHBjaCA9IDIxLCBiZyA9IGMoInJlZCIsICJncmVlbjMiLCAiYmx1ZSIpW3VuY2xhc3MoaXJpcyRTcGVjaWVzKV0pDQpgYGANCg0KYGBge3J9DQpCcFNsIDwtIGdncGxvdChpcmlzLCBhZXMoU3BlY2llcywgU2VwYWwuTGVuZ3RoLCBmaWxsID0gU3BlY2llcykpICsNCiAgICBnZW9tX2JveHBsb3QoKSArDQogICAgc2NhbGVfeV9jb250aW51b3VzKCJTZXBhbCBMZW5ndGggKGNtKSIsIGJyZWFrcyA9IHNlcSgwLCAzMCwgYnkgPSAuNSkpICsNCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAibm9uZSIpDQoNCkJwU3cgPC0gZ2dwbG90KGlyaXMsIGFlcyhTcGVjaWVzLCBTZXBhbC5XaWR0aCwgZmlsbCA9IFNwZWNpZXMpKSArDQogICAgZ2VvbV9ib3hwbG90KCkgKw0KICAgIHNjYWxlX3lfY29udGludW91cygiU2VwYWwgV2lkdGggKGNtKSIsIGJyZWFrcyA9IHNlcSgwLCAzMCwgYnkgPSAuNSkpICsNCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAibm9uZSIpDQoNCkJwUGwgPC0gZ2dwbG90KGlyaXMsIGFlcyhTcGVjaWVzLCBQZXRhbC5MZW5ndGgsIGZpbGwgPSBTcGVjaWVzKSkgKw0KICAgIGdlb21fYm94cGxvdCgpICsNCiAgICBzY2FsZV95X2NvbnRpbnVvdXMoIlBldGFsIExlbmd0aCAoY20pIiwgYnJlYWtzID0gc2VxKDAsIDMwLCBieSA9IC41KSkgKw0KICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIikNCg0KQnBQdyA8LSBnZ3Bsb3QoaXJpcywgYWVzKFNwZWNpZXMsIFBldGFsLldpZHRoLCBmaWxsID0gU3BlY2llcykpICsNCiAgICBnZW9tX2JveHBsb3QoKSArDQogICAgc2NhbGVfeV9jb250aW51b3VzKCJQZXRhbCBXaWR0aCAoY20pIiwgYnJlYWtzID0gc2VxKDAsIDMwLCBieSA9IC41KSkgKw0KICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIikNCg0KIyBQbG90IGFsbCB2aXN1YWxpemF0aW9ucw0KZ3JpZC5hcnJhbmdlKEJwU2wgKyBnZ3RpdGxlKCIiKSwNCiAgICBCcFN3ICsgZ2d0aXRsZSgiIiksDQogICAgQnBQbCArIGdndGl0bGUoIiIpLA0KICAgIEJwUHcgKyBnZ3RpdGxlKCIiKSwNCiAgICBucm93ID0gMiwNCiAgICB0b3AgPSB0ZXh0R3JvYigiSXJpcyBCb3ggUGxvdHMiLA0KICAgICAgICBncCA9IGdwYXIoZm9udHNpemUgPSAxNSkNCiAgICApDQopDQpgYGANCg0KYGBge3J9DQpsY19pcmlzIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KGlyaXNbMTo0XSksIGlyaXMkU3BlY2llcywNCiAgICBjbGFzc2lmaWVycyA9IGNsYXNzaWZpZXJzX0xTLCBtZWFzdXJlcyA9IG1lYXN1cmVzLA0KICAgIHR5cGUgPSAiZnJhY3Rpb24iLCBmcmFjcyA9IHNlcSgwLjEsIDAuOCwgMC4xKSwNCiAgICB0ZXN0X2ZyYWN0aW9uID0gMC41LCByZXBlYXRzID0gMw0KKQ0KDQpwbG90KGxjX2lyaXMpDQpgYGANCg0KIyMgU3BhbWJhc2UNCg0KIVtTcGFtIGVtYWlsXSguLi9pbWFnZXMvc3BhbS5wbmcpDQohW05vbiBzcGFtIGVtYWlsXSguLi9pbWFnZXMvbm9uc3BhbS5wbmcpDQoNCmBgYHtyfQ0Kc3BhbWJhc2UgPC0gcmVhZC5jc3YoIi4uL2RhdGEvc3BhbWJhc2UuY3N2IiwgaGVhZGVyID0gVFJVRSwgc2VwID0gIiwiKQ0Kc3BhbWJhc2UkY2xhc3MgPC0gYXMuZmFjdG9yKHNwYW1iYXNlJGNsYXNzKQ0Kc3BhbWJhc2Vbc2FtcGxlKG5yb3coc3BhbWJhc2UpLCAxMCksIF0NCmBgYA0KDQpgYGB7cn0NCmxjX3NwYW1iYXNlIDwtIExlYXJuaW5nQ3VydmVTU0woYXMubWF0cml4KHNwYW1iYXNlWzE6NTddKSwgc3BhbWJhc2UkY2xhc3MsDQogICAgY2xhc3NpZmllcnMgPSBjbGFzc2lmaWVyc19MUywgbWVhc3VyZXMgPSBtZWFzdXJlcywNCiAgICB0eXBlID0gImZyYWN0aW9uIiwgZnJhY3MgPSBzZXEoMC4xLCAwLjgsIDAuMSksDQogICAgdGVzdF9mcmFjdGlvbiA9IDAuNSwgcmVwZWF0cyA9IDMNCikNCg0KcGxvdChsY19zcGFtYmFzZSkNCmBgYA0K